
<!DOCTYPE HTML>
<html lang="" >
    <head>
        <meta charset="UTF-8">
        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
        <title>3.4 案例：贴吧图片 · GitBook</title>
        <meta http-equiv="X-UA-Compatible" content="IE=edge" />
        <meta name="description" content="">
        <meta name="generator" content="GitBook 3.2.3">
        
        
        
    
    <link rel="stylesheet" href="../gitbook/style.css">

    
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-highlight/website.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-search/search.css">
                
            
                
                <link rel="stylesheet" href="../gitbook/gitbook-plugin-fontsettings/website.css">
                
            
        

    

    
        
    
        
    
        
    
        
    
        
    
        
    

        
    
    
    <meta name="HandheldFriendly" content="true"/>
    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <link rel="apple-touch-icon-precomposed" sizes="152x152" href="../gitbook/images/apple-touch-icon-precomposed-152.png">
    <link rel="shortcut icon" href="../gitbook/images/favicon.ico" type="image/x-icon">

    
    <link rel="next" href="Json与JsonPath.html" />
    
    
    <link rel="prev" href="lxml.html" />
    

    </head>
    <body>
        
<div class="book">
    <div class="book-summary">
        
            
<div id="book-search-input" role="search">
    <input type="text" placeholder="Type to search" />
</div>

            
                <nav role="navigation">
                


<ul class="summary">
    
    

    

    
        
        
    
        <li class="chapter " data-level="1.1" data-path="../">
            
                <a href="../">
            
                    
                    前言
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2" data-path="正则表达式.html">
            
                <a href="正则表达式.html">
            
                    
                    第一章：正则表达式
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.2.1" data-path="正则表达式入门.html">
            
                <a href="正则表达式入门.html">
            
                    
                    1.1 正则表达式入门
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.2" data-path="Python中的正则表达式.html">
            
                <a href="Python中的正则表达式.html">
            
                    
                    1.2 Python中的正则表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.3" data-path="基本正则表达式.html">
            
                <a href="基本正则表达式.html">
            
                    
                    1.3 基本正则表达式
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.4" data-path="分组.html">
            
                <a href="分组.html">
            
                    
                    1.4 分组
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.5" data-path="先行断言.html">
            
                <a href="先行断言.html">
            
                    
                    1.5 先行断言
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.6" data-path="标记.html">
            
                <a href="标记.html">
            
                    
                    1.6 标记
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.7" data-path="替换与正则表达式编译.html">
            
                <a href="替换与正则表达式编译.html">
            
                    
                    1.7 替换与正则编译
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.2.8" data-path="正则表达式练习.html">
            
                <a href="正则表达式练习.html">
            
                    
                    1.8 正则表达式练习
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.3" data-path="Python爬虫学习目标.html">
            
                <a href="Python爬虫学习目标.html">
            
                    
                    第二章：爬虫与Requests
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.3.1" data-path="Requests的简单使用.html">
            
                <a href="Requests的简单使用.html">
            
                    
                    2.1 Requests的简单使用
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.3.2" data-path="深入了解Requests.html">
            
                <a href="深入了解Requests.html">
            
                    
                    2.2 深入了解Requets
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    
        <li class="chapter " data-level="1.4" data-path="结构化与非结构化.html">
            
                <a href="结构化与非结构化.html">
            
                    
                    第三章：非结构化与结构化数据提取
            
                </a>
            

            
            <ul class="articles">
                
    
        <li class="chapter " data-level="1.4.1" data-path="正则抓取.html">
            
                <a href="正则抓取.html">
            
                    
                    3.1 使用正则的爬虫
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.2" data-path="xpath与XML.html">
            
                <a href="xpath与XML.html">
            
                    
                    3.2 Xpath与XML类库
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.3" data-path="lxml.html">
            
                <a href="lxml.html">
            
                    
                    3.3 LXML库的使用
            
                </a>
            

            
        </li>
    
        <li class="chapter active" data-level="1.4.4" data-path="Xpath爬虫案例.html">
            
                <a href="Xpath爬虫案例.html">
            
                    
                    3.4 案例：贴吧图片
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.5" data-path="Json与JsonPath.html">
            
                <a href="Json与JsonPath.html">
            
                    
                    3.5 Json与Json数据获取
            
                </a>
            

            
        </li>
    
        <li class="chapter " data-level="1.4.6" data-path="实例：糗事百科网站抓取.html">
            
                <a href="实例：糗事百科网站抓取.html">
            
                    
                    3.6 糗事百科网站爬虫
            
                </a>
            

            
        </li>
    

            </ul>
            
        </li>
    

    

    <li class="divider"></li>

    <li>
        <a href="https://www.gitbook.com" target="blank" class="gitbook-link">
            Published with GitBook
        </a>
    </li>
</ul>


                </nav>
            
        
    </div>

    <div class="book-body">
        
            <div class="body-inner">
                
                    

<div class="book-header" role="navigation">
    

    <!-- Title -->
    <h1>
        <i class="fa fa-circle-o-notch fa-spin"></i>
        <a href=".." >3.4 案例：贴吧图片</a>
    </h1>
</div>




                    <div class="page-wrapper" tabindex="-1" role="main">
                        <div class="page-inner">
                            
<div id="book-search-results">
    <div class="search-noresults">
    
                                <section class="normal markdown-section">
                                
                                <h2 id="&#x6848;&#x4F8B;&#xFF1A;&#x4F7F;&#x7528;xpath&#x7684;&#x722C;&#x866B;">&#x6848;&#x4F8B;&#xFF1A;&#x4F7F;&#x7528;XPath&#x7684;&#x722C;&#x866B;</h2>
<p>&#x7528;XPath&#x6765;&#x505A;&#x4E00;&#x4E2A;&#x7B80;&#x5355;&#x7684;&#x722C;&#x866B;&#xFF0C;&#x6211;&#x4EEC;&#x5C1D;&#x8BD5;&#x722C;&#x53D6;&#x67D0;&#x4E2A;&#x8D34;&#x5427;&#x91CC;&#x7684;&#x6240;&#x6709;&#x5E16;&#x5B50;&#xFF0C;&#x5E76;&#x4E14;&#x5C06;&#x8BE5;&#x8FD9;&#x4E2A;&#x5E16;&#x5B50;&#x91CC;&#x6BCF;&#x4E2A;&#x697C;&#x5C42;&#x53D1;&#x5E03;&#x7684;&#x56FE;&#x7247;&#x4E0B;&#x8F7D;&#x5230;&#x672C;&#x5730;&#x3002;</p>
<pre><code class="lang-python"><span class="hljs-comment">#coding=utf-8</span>
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> lxml <span class="hljs-keyword">import</span> etree
<span class="hljs-keyword">import</span> json

<span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">Tieba</span>:</span>

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self,tieba_name)</span>:</span>
        self.tieba_name = tieba_name <span class="hljs-comment">#&#x63A5;&#x6536;&#x8D34;&#x5427;&#x540D;</span>
        <span class="hljs-comment">#&#x8BBE;&#x7F6E;&#x4E3A;&#x624B;&#x673A;&#x7AEF;&#x7684;UA</span>
        self.headers = {<span class="hljs-string">&quot;User-Agent&quot;</span>: <span class="hljs-string">&quot;Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1&quot;</span>}

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_total_url_list</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-string">&apos;&apos;&apos;&#x83B7;&#x53D6;&#x6240;&#x6709;&#x7684;urllist&apos;&apos;&apos;</span>
        url = <span class="hljs-string">&quot;https://tieba.baidu.com/f?kw=&quot;</span>+self.tieba_name+<span class="hljs-string">&quot;&amp;ie=utf-8&amp;pn={}&amp;&quot;</span>
        url_list = []
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> range(<span class="hljs-number">100</span>): <span class="hljs-comment">#&#x901A;&#x8FC7;&#x5FAA;&#x73AF;&#x62FC;&#x63A5;100&#x4E2A;url</span>
            url_list.append(url.format(i*<span class="hljs-number">50</span>))
        <span class="hljs-keyword">return</span> url_list <span class="hljs-comment">#&#x8FD4;&#x56DE;100&#x4E2A;url&#x7684;urllist</span>

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">parse_url</span><span class="hljs-params">(self,url)</span>:</span>
        <span class="hljs-string">&apos;&apos;&apos;&#x4E00;&#x4E2A;&#x53D1;&#x9001;&#x8BF7;&#x6C42;&#xFF0C;&#x83B7;&#x53D6;&#x54CD;&#x5E94;&#xFF0C;&#x540C;&#x65F6;etree&#x5904;&#x7406;html&apos;&apos;&apos;</span>
        print(<span class="hljs-string">&quot;parsing url:&quot;</span>,url)
        response = requests.get(url,headers=self.headers,timeout=<span class="hljs-number">10</span>) <span class="hljs-comment">#&#x53D1;&#x9001;&#x8BF7;&#x6C42;</span>
        html = response.text <span class="hljs-comment">#&#x83B7;&#x53D6;html&#x5B57;&#x7B26;&#x4E32;</span>
        html = etree.HTML(html) <span class="hljs-comment">#&#x83B7;&#x53D6;element &#x7C7B;&#x578B;&#x7684;html</span>
        <span class="hljs-keyword">return</span> html

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_title_href</span><span class="hljs-params">(self,url)</span>:</span>
        <span class="hljs-string">&apos;&apos;&apos;&#x83B7;&#x53D6;&#x4E00;&#x4E2A;&#x9875;&#x9762;&#x7684;title&#x548C;href&apos;&apos;&apos;</span>
        html = self.parse_url(url)
        li_temp_list = html.xpath(<span class="hljs-string">&apos;//li[@class=&quot;tl_shadow tl_shadow_new &quot;]&apos;</span>) <span class="hljs-comment">#&#x5206;&#x7EC4;&#xFF0C;&#x6309;&#x7167;li&#x6807;&#x7B7E;&#x5206;&#x7EC4;</span>
        total_items = []
        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> li_temp_list: <span class="hljs-comment">#&#x904D;&#x5386;&#x5206;&#x7EC4;</span>
            href = <span class="hljs-string">&quot;https://tieba.baidu.com&quot;</span>+i.xpath(<span class="hljs-string">&quot;./a/@href&quot;</span>)[<span class="hljs-number">0</span>] <span class="hljs-keyword">if</span> len(i.xpath(<span class="hljs-string">&quot;./a/@href&quot;</span>))&gt;<span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-keyword">None</span>
            text = i.xpath(<span class="hljs-string">&quot;./a/div[1]/span[1]/text()&quot;</span>)
            text = text[<span class="hljs-number">0</span>] <span class="hljs-keyword">if</span> len(text)&gt;<span class="hljs-number">0</span> <span class="hljs-keyword">else</span> <span class="hljs-keyword">None</span>
            item = dict(  <span class="hljs-comment">#&#x653E;&#x5165;&#x5B57;&#x5178;</span>
                href = href,
                text = text
            )
            total_items.append(item)
        <span class="hljs-keyword">return</span> total_items <span class="hljs-comment">#&#x8FD4;&#x56DE;&#x4E00;&#x4E2A;&#x9875;&#x9762;&#x6240;&#x6709;&#x7684;item</span>

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">get_img</span><span class="hljs-params">(self,url)</span>:</span>
        <span class="hljs-string">&apos;&apos;&apos;&#x83B7;&#x53D6;&#x4E00;&#x4E2A;&#x5E16;&#x5B50;&#x91CC;&#x9762;&#x7684;&#x6240;&#x6709;&#x56FE;&#x7247;&apos;&apos;&apos;</span>
        response = requests.get(url,timeout=<span class="hljs-number">10</span>) <span class="hljs-comment">#&#x53D1;&#x9001;&#x8BF7;&#x6C42;</span>
        html = etree.HTML(response.text) <span class="hljs-comment">#&#x8FD4;&#x56DE;elemet&#x7D2F;&#x5FC3;&#x7684;html&#xFF0C;&#x5177;&#x6709;xpath&#x65B9;&#x6CD5;</span>
        img_list = [img.xpath(<span class="hljs-string">&apos;./@src&apos;</span>)  <span class="hljs-keyword">for</span> img <span class="hljs-keyword">in</span> html.xpath(<span class="hljs-string">&apos;//img[@class=&quot;BDE_Image&quot;]&apos;</span>)]
        <span class="hljs-keyword">return</span> img_list

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">save_item</span><span class="hljs-params">(self,item)</span>:</span>
        <span class="hljs-string">&apos;&apos;&apos;&#x4FDD;&#x5B58;&#x4E00;&#x4E2A;item&apos;&apos;&apos;</span>
        <span class="hljs-keyword">with</span> open(<span class="hljs-string">&quot;teibatupian.txt&quot;</span>,<span class="hljs-string">&quot;a&quot;</span>) <span class="hljs-keyword">as</span> f:

            f.write(json.dumps(item,ensure_ascii=<span class="hljs-keyword">False</span>,indent=<span class="hljs-number">2</span>))
            f.write(<span class="hljs-string">&quot;\n&quot;</span>)

    <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">run</span><span class="hljs-params">(self)</span>:</span>
        <span class="hljs-comment">#1&#x3001;&#x627E;&#x5230;&#x4E86;url&#x89C4;&#x5F8B;&#xFF0C;url list</span>
        url_list = self.get_total_url_list()
        <span class="hljs-keyword">for</span> url <span class="hljs-keyword">in</span> url_list:
        <span class="hljs-comment">#2&#x3001;&#x904D;&#x5386;urllist &#x53D1;&#x9001;&#x8BF7;&#x6C42;&#xFF0C;&#x83B7;&#x5F97;&#x54CD;&#x5E94;&#xFF0C;etree&#x5904;&#x7406;html</span>
        <span class="hljs-comment"># 3&#x3001;&#x63D0;&#x53D6;title&#xFF0C;href</span>
            total_item = self.get_title_href(url)
            <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> total_item:
                href = item[<span class="hljs-string">&quot;href&quot;</span>]
                img_list = self.get_img(href) <span class="hljs-comment">#&#x83B7;&#x53D6;&#x5230;&#x4E86;&#x5E16;&#x5B50;&#x7684;&#x56FE;&#x7247;&#x5217;&#x8868;</span>
                item[<span class="hljs-string">&quot;img&quot;</span>] = img_list
                <span class="hljs-comment"># 4&#x3001;&#x4FDD;&#x5B58;&#x5230;&#x672C;&#x5730;</span>
                print(item)
                self.save_item(item)

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    tieba = Tieba(<span class="hljs-string">&quot;&#x732B;&quot;</span>)
    tieba.run()
</code></pre>

                                
                                </section>
                            
    </div>
    <div class="search-results">
        <div class="has-results">
            
            <h1 class="search-results-title"><span class='search-results-count'></span> results matching "<span class='search-query'></span>"</h1>
            <ul class="search-results-list"></ul>
            
        </div>
        <div class="no-results">
            
            <h1 class="search-results-title">No results matching "<span class='search-query'></span>"</h1>
            
        </div>
    </div>
</div>

                        </div>
                    </div>
                
            </div>

            
                
                <a href="lxml.html" class="navigation navigation-prev " aria-label="Previous page: 3.3 LXML库的使用">
                    <i class="fa fa-angle-left"></i>
                </a>
                
                
                <a href="Json与JsonPath.html" class="navigation navigation-next " aria-label="Next page: 3.5 Json与Json数据获取">
                    <i class="fa fa-angle-right"></i>
                </a>
                
            
        
    </div>

    <script>
        var gitbook = gitbook || [];
        gitbook.push(function() {
            gitbook.page.hasChanged({"page":{"title":"3.4 案例：贴吧图片","level":"1.4.4","depth":2,"next":{"title":"3.5 Json与Json数据获取","level":"1.4.5","depth":2,"path":"notes/Json与JsonPath.md","ref":"notes/Json与JsonPath.md","articles":[]},"previous":{"title":"3.3 LXML库的使用","level":"1.4.3","depth":2,"path":"notes/lxml.md","ref":"notes/lxml.md","articles":[]},"dir":"ltr"},"config":{"gitbook":"*","theme":"default","variables":{},"plugins":[],"pluginsConfig":{"highlight":{},"search":{},"lunr":{"maxIndexSize":1000000,"ignoreSpecialCharacters":false},"sharing":{"facebook":true,"twitter":true,"google":false,"weibo":false,"instapaper":false,"vk":false,"all":["facebook","google","twitter","weibo","instapaper"]},"fontsettings":{"theme":"white","family":"sans","size":2},"theme-default":{"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"},"showLevel":false}},"structure":{"langs":"LANGS.md","readme":"README.md","glossary":"GLOSSARY.md","summary":"SUMMARY.md"},"pdf":{"pageNumbers":true,"fontSize":12,"fontFamily":"Arial","paperSize":"a4","chapterMark":"pagebreak","pageBreaksBefore":"/","margin":{"right":62,"left":62,"top":56,"bottom":56}},"styles":{"website":"styles/website.css","pdf":"styles/pdf.css","epub":"styles/epub.css","mobi":"styles/mobi.css","ebook":"styles/ebook.css","print":"styles/print.css"}},"file":{"path":"notes/Xpath爬虫案例.md","mtime":"2019-05-05T15:52:28.981Z","type":"markdown"},"gitbook":{"version":"3.2.3","time":"2019-05-08T02:50:45.307Z"},"basePath":"..","book":{"language":""}});
        });
    </script>
</div>

        
    <script src="../gitbook/gitbook.js"></script>
    <script src="../gitbook/theme.js"></script>
    
        
        <script src="../gitbook/gitbook-plugin-search/search-engine.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-search/search.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/lunr.min.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-lunr/search-lunr.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-sharing/buttons.js"></script>
        
    
        
        <script src="../gitbook/gitbook-plugin-fontsettings/fontsettings.js"></script>
        
    

    </body>
</html>

